package service;

import java.io.IOException;
import java.util.ArrayList;

import org.apache.log4j.Logger;

import util.FetchUtil;
import crawler.CrawlerDocument;
import crawler.DocumentPool;
import crawler.XmlExtractor;

public class Fetcher implements Runnable {
	
	private final Logger logger = Logger.getLogger(Fetcher.class);
	
	private DocumentPool _urlPool = null;
	private DocumentPool _docPool = null;
	private Deduper _deduper = null;
	
	public Fetcher(DocumentPool urlPool, DocumentPool docPool){
		_urlPool = urlPool;
		_docPool = docPool;
		_deduper = Deduper.getInstance();
	}

	@Override
	public void run() {
		while(true){
			try {
				CrawlerDocument doc = _urlPool.take();
				
				FetchUtil.fetch(doc);	
				
				XmlExtractor extractor = new XmlExtractor();
				
				extractor.Extract(doc);
				
				ArrayList<CrawlerDocument> outlinks = doc.getOutlinks();
				
				logger.info("Size of outlink : " + outlinks.size());
				
				for(int i = 0; i < outlinks.size(); ++i){
					CrawlerDocument extDoc = outlinks.get(i);	
					logger.debug("Doc extracted : " + extDoc.getFeature("link"));					
					if (!this._deduper.isDuplicated(extDoc)){
						_docPool.put(extDoc);
					}
				}	
			} catch (InterruptedException e) {
				logger.error("{}", e);
			} catch (IOException e){
				logger.error("{}", e);
			}
		}

	}

}
